package com.spider.silence.downloader;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.spider.silence.abstarct.downloader.AbstractCommonDownLoader;
import com.spider.silence.task.AiKaForumTask;
import com.spider.silence.task.Task;
import com.spider.silence.task.manager.CommonTaskManager;
import com.spider.silence.utils.*;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.File;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;

/**
 * @author 夏茂轩
 * @package com.spider.silence.downloader
 * @Date 2017/3/8 15:45
 */
public class AiKaForumDownLoader extends AbstractCommonDownLoader {
    private Logger logger = Logger.getLogger(AiKaForumDownLoader.class);
    private Map<String,File> files=null;
    private static final  String HOST="http://www.xcar.com.cn/bbs/";
    public Map<String,File> getFiles(){

        return files;
    }

    public void generateTask() {
        ExecutorService pool = Executors.newFixedThreadPool(10);
        try {
            org.dom4j.Document document = XmlUtil.getConf("META-INF/AiKaForum-conf.xml");
            final List<org.dom4j.Element> elements = document.getRootElement().elements("forum");
            for (int forumPageSize = 0; forumPageSize < elements.size(); forumPageSize++) {
                final int finalForumPageSize = forumPageSize;
                Thread thread = new Thread(new Runnable() {
                    @Override
                    public void run() {
                        org.dom4j.Element element = elements.get(finalForumPageSize);
                        String url = element.element("forum-url").getText();
                        String crawl_all = element.element("crawl-all").getText();
                        //添加第一页任务
                        try {
                            Document forumDoc = get(url,true);
                            Element container=forumDoc.select(".table-section").last();
                            Elements aTags=container.select(".thenomal>a");
                            for(int i=0;i<aTags.size();i++){
                                Task task =new AiKaForumTask(aTags.get(i).absUrl("href"));
                                CommonTaskManager.add("AiKaForumTask",task);
                            }
                        } catch (Exception e) {
                            e.printStackTrace();
                        }
                        if (crawl_all.equals("true")) {
                            try {
                                for (int page = 2; page < 1001; page++) {
                                    String getUrl=url+"&page="+page;
                                    Document forumDoc = get(getUrl,true);
                                    Element container=forumDoc.select(".table-section").last();
                                    Elements aTags=container.select(".thenomal>a");
                                    for(int i=0;i<aTags.size();i++){
                                        Task task =new AiKaForumTask(aTags.get(i).absUrl("href"));
                                        CommonTaskManager.add("AiKaForumTask",task);
                                    }
                                }
                            } catch (Exception e) {
                                e.printStackTrace();
                            }
                        }
                    }
                });
                pool.execute(thread);
            }
        } catch (Exception e) {
            e.printStackTrace();
            logger.error(e.getMessage(), e);
        } finally {
            pool.shutdown();
        }
    }

    public JSONObject crawlItem(String url) {
        files=new ConcurrentHashMap<>();
        JSONObject item = new JSONObject();
        JSONObject data = new JSONObject();
        int page = 1;
        int threadNum = Integer.valueOf(PropertiesUtil.getVal("AiKa.ReplyThreadCount"));
        ExecutorService executorService = Executors.newFixedThreadPool(threadNum);
        try {
            Document document = get(url,true);
            Elements msg_boxs = document.select(".t_row");
            //解析发帖信息
            Element publish_box = msg_boxs.first();
            Element publish_user_box = publish_box.select(".t_user").first();
            String publish_user_name = publish_user_box.select("a[rel='nofollow']").first().text();
            String publish_user_id = getUrlParams(publish_user_box.select("a[rel='nofollow']").first().attr("href")).get("uid").toString();
            String publish_user_photo = publish_user_box.select(".avatar img").first().absUrl("src");
            Element publish_msg_box = publish_box.select(".t_msg").first();
            String publish_time = getTime(publish_msg_box.select("tbody > tr:nth-child(1) > td > div > div:nth-child(2)").text());
            String content = publish_msg_box.select("tr").get(1).text();
            JSONArray img_urls=new JSONArray();
            Elements imgElements1=publish_msg_box.select("tr").get(1).select("img");
            //Elements imgElements2=publish_msg_box.select("tr").get(1).select("img[smilieid]");
            if(imgElements1.size()>0){
                for(int imgIndex=0;imgIndex<imgElements1.size();imgIndex++){
                    Element imgElement=imgElements1.get(imgIndex);
                    String img_url=imgElement.absUrl("src");
                    if(img_url.isEmpty()){
                        continue;
                    }
                    img_urls.add(img_url);
                }
            }
           /* if(imgElements2.size()>0){
                for(int imgIndex=0;imgIndex<imgElements2.size();imgIndex++){
                    Element imgElement=imgElements2.get(imgIndex);
                    String img_url=imgElement.absUrl("src");
                    img_urls.add(img_url);
                }
            }*/
            String title = document.title();
            String tid = getUrlParams(url).get("tid").toString();
            Element count_box = document.select("#showPic").first();
            String countString = count_box.text().replace("回复：", "").replace("查看：", "#");
            String[] counts = countString.split("#");
            int reply_num = Integer.parseInt(counts[0].replaceAll("\\s\\S", ""));
            int read_num = Integer.parseInt(counts[1].replaceAll("\\s\\S", ""));
            data.put("reply_count",reply_num);
            data.put("read_count",read_num);
            data.put("title", (title));
            data.put("url",url);
            data.put("publish_time",publish_time);
            data.put("reproduce_count",0);
            data.put("like_count",0);
            data.put("dislike_count",0);
            data.put("content",FileUtils.filterImg(content));
            data.put("img_urls",img_urls);
            data.put("publish_user_id",publish_user_id);
            data.put("publish_user", (publish_user_name));
            data.put("publish_user_photo",publish_user_photo);
            data.put("id",tid);

            long tsp= DateUtils.dateToTimestamp( DateUtils.parseDate(publish_time)).getTime();
            String filePrefix="AiKaForum";//文件前缀
            String fileName=filePrefix+"_"+tsp+"_"+publish_user_id+"_"+tid;
            String publishDate=DateUtils.formatDate(DateUtils.parseDate(publish_time)).replace("-","-");
            String speechesId="AiKaForum_speeches_"+publishDate+"_"+fileName;
            File file= FileUtils.createTempFile(speechesId+"_pageNo_1",document.html().toString());
            files.put(speechesId+"_pageNo_1",file);
            //帖子首页的文件

            Date now = new Date();
            SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
            data.put("spider_time",sdf.format(now));
            Element pageElement = document.select(".FpageNum").first();
            Element lastPageElement = pageElement.select("a:contains(下一页)").first();
            Element maxPageElement = lastPageElement.previousElementSibling();
            int maxPage = Integer.parseInt(maxPageElement.text());
            JSONArray reply_nodes=new JSONArray();
            for (int i = 1; i < msg_boxs.size(); i++) {
                JSONObject reply=new JSONObject();
                Element reply_box = msg_boxs.get(i);
                String msg_id=reply_box.attr("id");
                Element reply_user_box = reply_box.select(".t_user").first();
                String reply_user_name = reply_user_box.select("a[rel='nofollow']").first().text();
                String reply_user_id = getUrlParams(reply_user_box.select("a[rel='nofollow']").first().attr("href")).get("uid").toString();
                String reply_user_photo = reply_user_box.select(".avatar img").first().absUrl("src");
                Element reply_msg_box = reply_box.select(".t_msg").first();
                String reply_time = getTime(reply_msg_box.select(" .tbody > tr:nth-child(1) > td > div > div:nth-child(2)").text());
                String reply_content = reply_msg_box.select("tr").get(1).text();
                JSONArray reply_img_urls=new JSONArray();
                Elements replyImgElements1=reply_msg_box.select("tr").get(1).select("img");//图片
                //Elements replyImgElements2=reply_msg_box.select("tr").get(1).select("img[smilieid]");//表情包
                if(replyImgElements1.size()>0){
                    for(int imgIndex=0;imgIndex<replyImgElements1.size();imgIndex++){
                        Element imgElement=replyImgElements1.get(imgIndex);
                        String img_url=imgElement.absUrl("src");
                        if(img_url.isEmpty()){
                            continue;
                        }
                        reply_img_urls.add(img_url);
                    }
                }
               /* if(replyImgElements2.size()>0){
                    for(int imgIndex=0;imgIndex<replyImgElements2.size();imgIndex++){
                        Element imgElement=replyImgElements2.get(imgIndex);
                        String img_url=imgElement.absUrl("src");
                        reply_img_urls.add(img_url);
                    }
                }*/
                String reply_url=url+"#"+msg_id;
                reply.put("publish_user",(reply_user_name));
                reply.put("publish_user_id",reply_user_id);
                reply.put("publish_user_photo",reply_user_photo);
                reply.put("content",reply_content);
                reply.put("img_urls",reply_img_urls);
                reply.put("publish_time",reply_time);
                reply.put("url",reply_url);
                reply.put("parent_id",tid);
                reply.put("ancestor_id",tid);
                reply_nodes.add(reply);
            }
            //解析发帖信息
            if (maxPage >= 2) {
                List<FutureTask<JSONArray>> futureTasks = new ArrayList<FutureTask<JSONArray>>();
                ArrayList<ArrayList<Integer>> splitPages = CommonUtils.splitPages(2, maxPage, threadNum);
                for (int i = 0; i < splitPages.size(); i++) {
                    ReplyCrawler crawler  = new ReplyCrawler(splitPages.get(i),url,speechesId);
                    FutureTask<JSONArray> futureTask = new FutureTask<JSONArray>(crawler);
                    futureTasks.add(futureTask);
                    executorService.submit(futureTask);
                }
                for (FutureTask<JSONArray> futureTask : futureTasks) {
                    JSONArray replyArrays=futureTask.get();
                    for(int i = 0;i<replyArrays.size();i++){
                        JSONObject object = replyArrays.getJSONObject(i);
                        reply_nodes.add(object);
                    }
                }
            }
            data.put("reply_nodes",reply_nodes);
        } catch (Exception e) {
            e.printStackTrace();
            logger.error(e.getMessage(), e);
        } finally {
            executorService.shutdown();
        }
        item.put("data",data);
        return  item;
    }

    private class ReplyCrawler implements Callable<JSONArray> {
        private ArrayList<Integer> pages;
        private String url;
        private String speeches_id;
        @Override
        public JSONArray call() throws Exception {
            AiKaForumDownLoader downLoader = new AiKaForumDownLoader();
            JSONArray array =new JSONArray();
            try {
                for(int k=0;k<pages.size();k++){
                    String getUrl=url+"&page="+pages.get(k);
                    try {
                        Document document = downLoader.get(getUrl,true);
                        File file=FileUtils.createTempFile(speeches_id+"_pageNo_"+pages.get(k),document.html().toString());
                        files.put(speeches_id+"_pageNo_"+pages.get(k),file);
                        Elements msg_boxs = document.select(".t_row");
                        for (int i = 0; i < msg_boxs.size(); i++) {
                            JSONObject reply=new JSONObject();
                            Element reply_box = msg_boxs.get(i);
                            String msg_id=reply_box.attr("id");
                            Element reply_user_box = reply_box.select(".t_user").first();
                            String reply_user_name = reply_user_box.select("a[rel='nofollow']").first().text();
                            String reply_user_id = getUrlParams(reply_user_box.select("a[rel='nofollow']").first().attr("href")).get("uid").toString();
                            String reply_user_photo = reply_user_box.select(".avatar img").first().absUrl("src");
                            Element reply_msg_box = reply_box.select(".t_msg").first();
                            String reply_time = getTime(reply_msg_box.select(" .tbody > tr:nth-child(1) > td > div > div:nth-child(2)").text());
                            String reply_content = reply_msg_box.select("tr").get(1).text();
                            JSONArray reply_img_urls=new JSONArray();
                            Elements replyImgElements1=reply_msg_box.select("tr").get(1).select("img");//图片
                            //Elements replyImgElements2=reply_msg_box.select("tr").get(1).select("img[smilieid]");//表情包
                            if(replyImgElements1.size()>0){
                                for(int imgIndex=0;imgIndex<replyImgElements1.size();imgIndex++){
                                    Element imgElement=replyImgElements1.get(imgIndex);
                                    String img_url=imgElement.absUrl("src");
                                    if(img_url.isEmpty()){
                                        continue;
                                    }
                                    reply_img_urls.add(img_url);
                                }
                            }
                            /*if(replyImgElements2.size()>0){
                                for(int imgIndex=0;imgIndex<replyImgElements2.size();imgIndex++){
                                    Element imgElement=replyImgElements2.get(imgIndex);
                                    String img_url=imgElement.absUrl("src");
                                    reply_img_urls.add(img_url);
                                }
                            }*/
                            String reply_url=getUrl+"#"+msg_id;
                            reply.put("publish_user",(reply_user_name));
                            reply.put("publish_user_id",reply_user_id);
                            reply.put("publish_user_photo",reply_user_photo);
                            reply.put("content",(FileUtils.filterImg(reply_content)));
                            reply.put("img_urls",reply_img_urls);
                            reply.put("publish_time",reply_time);
                            reply.put("url",reply_url);
                            reply.put("parent_id",downLoader.getUrlParams(url).get("tid").toString());
                            reply.put("ancestor_id",downLoader.getUrlParams(url).get("tid").toString());
                            array.add(reply);
                        }
                    } catch (Exception e) {
                        e.printStackTrace();
                        logger.error(e.getMessage(),e);
                    }
                }
            } catch (Exception e) {
                logger.error(e.getMessage(),e);
                e.printStackTrace();
            }
            return array;
        }

        public ReplyCrawler(ArrayList<Integer> pages, String url,String speeches_id) {
            this.pages = pages;
            this.url = url;
            this.speeches_id=speeches_id;
        }
    }
}
